from database.db_handler import MysqlHander
from common.my_http import MyHttp
from common.my_time import MyTime
from common.my_baidu import MyBaidu
import urllib.request
from bs4 import BeautifulSoup
import glob
import jieba
import re
import json
import time
import jieba
import re
import os
import json
from database.db_business import DbBusiness

remove = ['明天', '继续', '大盘', '缺口', '时候', '资讯', '开始', '涨幅', '利好', '主力', '就是', '一个', '垃圾', '以下', '你们', '大笑', '这么', '股票', '恶心', '什么', '意思', '怎么', '反包', '跌停', '下午', '融资', '今天', '股份', '有限公司', '关于', '剩余', '部分', '公告', '第二次', '临时', '散户', '资金', '高开', '昨天', '还会', '上市', '分钟', '现在', '卧槽', '发行', '估值', '两个'] # ['明天','资讯','什么','公告','今天','利好','垃圾','继续','大盘','时候','开始','主力','就是','以下','大笑','这么','股票','意思','跌停','关于','第二次', '散户','卧槽','两个','缺口','涨幅','一个','你们','恶心','怎么','反包','下午','融资','股份','有限公司','部分','临时','资金','高开','昨天','还会','上市','分钟','现在','估值','标题','一次','空头','多么','操作','刚好','清仓','牛逼亏','三个','不怎么','利索','快速','回调','买盘','消耗','大单','不断','逻辑','上去','主卖','扶不起','阿斗','收到','报道','以上','不卖','狂卖','是不是','庄家','送出','一定','要收','没见','拉鸡','垃圾股']

file = open('./out/' + os.path.split(__file__)[-1].split(".")[0], 'w', encoding = 'utf-8')
class BaiduRedian:
    def __init__(self):
        self.db = MysqlHander("config.ini")
        self.redian_all = []
        self.end_date = MyTime.forward_relative_date(7)
        self.business = DbBusiness()
    
    def get_TF(self, words, n):
        tf_dic = {}
        tf_data = {}
        for w in words:
            tf_dic[w] = tf_dic.get(w, 0) + 1
        for word, tf in sorted(tf_dic.items(), key = lambda x : x[1], reverse = True):
            if tf > n and word in self.redian_all:
                tf_data[word] = tf
            elif tf > n:
                file.write(word + "\n")

        return tf_data

    def get_redian(self, code, name):
        words_all = []
        #更新前需要先删除旧数据，暂未处理
        next_page = True
        for i in range(1, 10):
            print(i)
            #time.sleep(2)
            try:
                #time.sleep(1)
                if next_page == False:
                    break
                url = MyBaidu.get_url_order_by_time([name], i)
                res = []
                proxy = self.business.query_proxy()
                if len(proxy) > 0:
                    res = MyBaidu.get_baidu_data_by_proxy(url, proxy)
                else:
                    res = MyBaidu.get_baidu_data(url)
                for r in res:
                    data_date = MyBaidu.calc_date(r["date"])
                    if data_date < self.end_date:
                        next_page = False
                        break
                    data_title = r["title"]
                    if MyBaidu.check_title(data_title, [[name]]) == False:
                        continue
                    data_site = r["site"]
                    data_url = r["url"]
                    #words = [x for x in jieba.cut(data_title) if self.is_contain_chinese(x) and x in self.redian_all]
                    words = [x for x in jieba.cut(data_title) if self.is_contain_chinese(x)]
                    #print(str(words))
                    words_all.extend(words)
                    #print(str(words_all))
            except Exception as e:
                print(str(e))
                pass
        datas = self.get_TF(words_all, 3)
        print(datas)
        for data in datas:
            self.add_redian_relation(code, data)
    
    def is_contain_chinese(self, check_str):
        for ch in check_str: 
            if u'\u4e00' <= ch <= u'\u9fff': 
                return True
        return False
    
    def get_redian_info(self):
        sql = "select Fattribute_name from attribute_data where Frelation_id = 2005"
        redian_data = self.db.query_sql(sql)
        for d in redian_data:
            print("a")
            self.redian_all.append(d[0])
        print(self.redian_all)

    def gupiao_redian(self, sql):
        print(sql)
        gupiao_data = self.db.query_sql(sql)
        self.get_redian_info()
        for gupiao in gupiao_data:
            self.get_redian(gupiao[0], gupiao[1])
            #self.save_redian_data(gupiao[0], redian)
        #self.save_redian_file()
    
    def add_redian_relation(self, code, word):
        sql = "insert into relation_data_02 (Fsrc_code, Fsrc_type, Frelation_id, Fattribute_name, Forder_tag, Fcreate_time, Fmodify_time)"
        sql += "values ('" + code +"',0,2005,'"+word+"',0,now(),now());"
        print(sql)
        #file_out.write(sql + "\n")
        try:
            self.db.execute_not_safe(sql)
        except Exception as e:
            print(str(e))
            pass

    
    def save_redian_data(self, code, redian):
        print(code + ":" + redian)
        pass

if __name__ == '__main__':
    code = BaiduRedian()
    code.gupiao_redian("select Fcode,Fname from entity_gupiao;")
    file.close()